"""
微博热搜爬虫
"""

from typing import List, Dict, Any
import re
import json
import time
from bs4 import BeautifulSoup

from .base import BaseScraper
from ..core.config import get_settings
from ..core.logger import app_logger

settings = get_settings()


class WeiboScraper(BaseScraper):
    """微博热搜爬虫"""
    
    def __init__(self):
        super().__init__()
        # 更新User-Agent以更好地模拟浏览器
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
            'Referer': 'https://s.weibo.com/top/summary',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
            'Accept-Encoding': 'gzip, deflate, br',
            'Connection': 'keep-alive',
            'Upgrade-Insecure-Requests': '1',
        }
        self.hot_url = "https://s.weibo.com/top/summary"
    
    @property
    def name(self) -> str:
        return "微博热搜"
    
    @property
    def description(self) -> str:
        return "抓取微博实时热搜榜"
    
    async def scrape(self) -> List[Dict[str, Any]]:
        """抓取微博热搜数据"""
        
        app_logger.info("开始抓取微博热搜...")
        
        try:
            # 使用真实数据抓取
            trends = await self._scrape_weibo_hot_real()
            
            app_logger.info(f"微博热搜抓取完成，共 {len(trends)} 条")
            return trends
            
        except Exception as e:
            app_logger.error(f"微博热搜抓取失败: {str(e)}")
            return []
    
    async def _scrape_weibo_hot_real(self) -> List[Dict[str, Any]]:
        """抓取微博热搜的真实实现"""
        try:
            # 获取页面内容
            html = await self.fetch(self.hot_url)
            if not html:
                raise Exception("获取微博热搜页面失败")
            
            # 检查是否包含热搜数据
            if "top_summitlist" not in html and "热搜" not in html and "toplist" not in html:
                app_logger.warning("微博页面可能没有正确加载热搜数据")
                # 尝试其他可能的URL
                return await self._try_alternative_urls()
            
            # 尝试多种方式解析数据
            trends = await self._parse_trends_from_html(html)
            
            # 如果没有获取到数据，使用模拟数据
            if not trends:
                app_logger.warning("未获取到真实微博热搜数据，使用模拟数据")
                return await self._get_mock_data()
            
            return trends
            
        except Exception as e:
            app_logger.error(f"抓取微博热搜真实数据失败: {e}")
            # 出错时返回模拟数据
            return await self._get_mock_data()
    
    async def _try_alternative_urls(self) -> List[Dict[str, Any]]:
        """尝试其他可能的URL"""
        alternative_urls = [
            "https://s.weibo.com/top/summary?cate=realtimehot",
            "https://s.weibo.com/top/summary?cate=socialevent",
            "https://s.weibo.com/top/summary?cate=total"
        ]
        
        for url in alternative_urls:
            try:
                html = await self.fetch(url)
                if html and ("热搜" in html or "toplist" in html):
                    trends = await self._parse_trends_from_html(html)
                    if trends:
                        return trends
            except Exception as e:
                app_logger.warning(f"尝试备用URL {url} 失败: {e}")
                continue
        
        return await self._get_mock_data()
    
    async def _parse_trends_from_html(self, html: str) -> List[Dict[str, Any]]:
        """从HTML中解析热搜数据"""
        trends = []
        
        try:
            soup = BeautifulSoup(html, 'html.parser')
            
            # 方法1: 查找包含热搜的script标签
            script_tags = soup.find_all('script')
            for script in script_tags:
                if script.string and ('topData' in script.string or '热搜' in script.string):
                    # 尝试提取JSON数据
                    try:
                        # 查找类似 var topData = [...] 的结构
                        json_match = re.search(r'var\s+topData\s*=\s*(\[.*?\]);', script.string, re.DOTALL)
                        if json_match:
                            json_str = json_match.group(1)
                            # 简单清理JSON字符串
                            json_str = re.sub(r'(\w+):', r'"\1":', json_str)  # 为key添加引号
                            data = json.loads(json_str)
                            for i, item in enumerate(data[:20], 1):  # 取前20条
                                title = item.get("title", "") or item.get("word", "")
                                if title:
                                    heat_score = item.get("hot", 0) or item.get("num", 0)
                                    # 如果热度为0，根据排名估算
                                    if heat_score == 0:
                                        heat_score = max(1, 21 - i) * 75000
                                        
                                    trends.append({
                                        "title": title,
                                        "description": item.get("desc", "") or item.get("content", ""),
                                        "rank": i,
                                        "heat_score": heat_score,
                                        "url": f"https://s.weibo.com/weibo?q={title}"
                                    })
                            if trends:
                                return trends
                    except Exception as e:
                        app_logger.warning(f"解析微博JSON数据失败: {e}")
            
            # 方法2: 查找表格形式的热搜
            trend_items = (soup.select('tbody tr') or 
                          soup.select('.list_a li') or 
                          soup.select('.card-wrap') or
                          soup.select('[node-type="hotlistItem"]'))
            
            if trend_items:
                for i, item in enumerate(trend_items[:20], 1):  # 取前20条
                    try:
                        # 提取标题
                        title_elem = (item.select_one('a') or 
                                    item.select_one('.title') or 
                                    item.select_one('h3') or
                                    item.select_one('.name') or
                                    item.select_one('[title]'))
                        
                        title = ""
                        if not title_elem:
                            # 尝试从属性获取
                            title = item.get('title', '') or item.get('data-title', '')
                            if not title:
                                # 尝试从文本获取
                                text = item.get_text().strip()
                                if text and len(text) > 1:
                                    lines = text.split('\n')
                                    title = lines[0].strip() if lines else ""
                        else:
                            title = title_elem.get_text().strip()
                        
                        if not title or len(title) < 2:
                            continue
                            
                        # 提取链接
                        link_elem = title_elem or item.select_one('a') or item.find('a')
                        link = ""
                        if link_elem and link_elem.get('href'):
                            link = link_elem.get('href', '')
                            if link.startswith('//'):
                                link = 'https:' + link
                            elif link.startswith('/'):
                                link = 'https://s.weibo.com' + link
                            elif not link.startswith('http'):
                                link = 'https://s.weibo.com/weibo?q=' + title
                        else:
                            link = 'https://s.weibo.com/weibo?q=' + title
                        
                        # 提取热度
                        hot_elem = (item.select_one('.hot') or 
                                  item.select_one('.num') or 
                                  item.select_one('.td-hot-value') or
                                  item.select_one('.heat') or
                                  item.select_one('[data-num]'))
                        
                        heat_score = 0
                        if hot_elem:
                            hot_text = hot_elem.get_text().strip()
                            # 提取数字
                            hot_match = re.search(r'(\d+)', hot_text)
                            if hot_match:
                                heat_score = int(hot_match.group(1)) * 10000  # 转换为近似热度值
                        else:
                            # 从属性获取热度
                            hot_attr = item.get('data-num', '') or item.get('data-hot', '')
                            if hot_attr:
                                try:
                                    heat_score = int(hot_attr) * 10000
                                except ValueError:
                                    pass
                        
                        # 如果热度为0，根据排名估算
                        if heat_score == 0:
                            heat_score = max(1, 21 - i) * 75000  # 第1名约150万热度，第20名约7.5万
                        
                        trends.append({
                            "title": title,
                            "description": "",
                            "rank": i,
                            "heat_score": heat_score,
                            "url": link
                        })
                    except Exception as e:
                        app_logger.warning(f"解析单条微博热搜失败: {e}")
                        continue
            
            # 方法3: 如果还是没有数据，尝试直接解析页面中的JSON数据
            if not trends:
                # 查找页面中可能包含热榜数据的JSON
                json_pattern = r'\{"word":"[^"]*","num":\d+,"flag":[^}]*\}'
                json_matches = re.findall(json_pattern, html)
                for i, json_str in enumerate(json_matches[:20], 1):
                    try:
                        data = json.loads(json_str)
                        title = data.get("word", "")
                        if title:
                            heat_score = data.get("num", 0)
                            # 如果热度为0，根据排名估算
                            if heat_score == 0:
                                heat_score = max(1, 21 - i) * 75000
                                
                            trends.append({
                                "title": title,
                                "description": data.get("desc", "") or data.get("content", ""),
                                "rank": i,
                                "heat_score": heat_score,
                                "url": f"https://s.weibo.com/weibo?q={title}"
                            })
                    except Exception as e:
                        app_logger.warning(f"解析微博页面JSON数据失败: {e}")
                        continue
            
            return trends
            
        except Exception as e:
            app_logger.error(f"解析微博热搜HTML失败: {e}")
            return []
    
    async def _get_mock_data(self) -> List[Dict[str, Any]]:
        """获取模拟数据"""
        
        mock_trends = [
            {
                "title": "AI技术发展新突破",
                "description": "人工智能技术迎来重大突破，多个领域应用前景广阔",
                "rank": 1,
                "heat_score": 1500000,
                "url": "https://s.weibo.com/weibo?q=%23AI技术发展新突破%23"
            },
            {
                "title": "新能源汽车销量创新高", 
                "description": "新能源汽车市场持续火热，销量屡创新高",
                "rank": 2,
                "heat_score": 1200000,
                "url": "https://s.weibo.com/weibo?q=%23新能源汽车销量创新高%23"
            },
            {
                "title": "太空探索最新进展",
                "description": "太空探索技术取得重要进展，为人类探索宇宙奠定基础",
                "rank": 3,
                "heat_score": 980000,
                "url": "https://s.weibo.com/weibo?q=%23太空探索最新进展%23"
            }
        ]
        
        return mock_trends